package com.test.mess;

import java.io.*;
import java.net.URL;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;

/**
 * @author fs
 */
public class DlPage {

    public static void main(String[] args) throws Throwable {
        dl();
    }

    public static void dl() throws Exception {
        BufferedReader br = new BufferedReader(new FileReader("d:\\jmeter.txt"));
        List<String> urls = br.lines()
                .filter(l -> l.contains("href=\""))
                .map(o -> "http://jmeter.apache.org/usermanual/" + o.substring(o.lastIndexOf("href") + 6, o.indexOf(".html") + 5))
                .collect(Collectors.toList());
//                .parallelStream().filter(o -> !o.contains(".."))
//                .forEach(System.out::println);
        Set urlSet = new HashSet<>();
        urls.parallelStream().forEach(o -> {
            if (urlSet.contains(o)) {
                return;
            }
            System.out.println(o);
            String line;
            try {
                URL url = new URL(o);
                BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream()));
                BufferedWriter writer = new BufferedWriter(new FileWriter("d:/jmeter/" + o.substring(o.lastIndexOf("/"))));
                while ((line = reader.readLine()) != null) {
                    writer.write(line);
                    writer.newLine();
                }
                reader.close();
                writer.close();
            } catch (Exception e) {
                e.printStackTrace();
            } finally {

            }

            urlSet.add(o);
        });
    }
}
